{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch \n",
    "import torch.nn as nn\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MLP(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(MLP, self).__init__()\n",
    "        self.first_layer = nn.Linear(100,50)\n",
    "        self.second_layer = nn.Linear(50, 1)\n",
    "    def forward(self, x):\n",
    "        x = nn.functional.relu(self.first_layer(x))\n",
    "        x = self.second_layer(x)\n",
    "        return x\n",
    "mlp = MLP()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import TensorDataset\n",
    "x = torch.randn(1000, 100).to(device)\n",
    "y = (torch.rand(1000)>0.5).int().float().to(device) # When feeding the data to optimizer, it only accepts floats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import TensorDataset, DataLoader\n",
    "dataset = TensorDataset(x, y)\n",
    "dataloader = DataLoader(dataset, batch_size=100, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 0.3349, -0.3357,  1.5577,  ...,  1.2535, -0.8741,  1.1964],\n",
      "        [ 0.5997, -0.1040, -0.0421,  ...,  2.0984, -1.6125, -0.2931],\n",
      "        [ 0.9497, -0.1843, -2.0102,  ..., -0.4697,  1.1201,  1.2877],\n",
      "        ...,\n",
      "        [ 0.1315, -0.4559, -1.6739,  ...,  1.5253, -1.2748, -1.0905],\n",
      "        [ 0.4752,  0.9459, -0.5876,  ...,  2.5902, -0.0187, -1.1862],\n",
      "        [-0.2458, -0.7106, -1.1863,  ...,  0.4584, -0.0682, -1.2806]],\n",
      "       device='cuda:0')\n",
      "tensor([1., 1., 0., 0., 1., 1., 0., 0., 0., 0., 1., 0., 0., 0., 1., 0., 1., 0.,\n",
      "        1., 0., 1., 0., 1., 0., 0., 0., 0., 0., 1., 1., 0., 0., 0., 1., 0., 1.,\n",
      "        0., 1., 1., 1., 0., 0., 0., 0., 1., 1., 1., 1., 0., 1., 1., 0., 0., 1.,\n",
      "        1., 0., 1., 1., 1., 0., 1., 1., 1., 0., 0., 1., 1., 1., 1., 0., 0., 1.,\n",
      "        1., 1., 0., 0., 0., 1., 1., 0., 0., 1., 1., 0., 0., 1., 1., 0., 0., 1.,\n",
      "        1., 0., 0., 1., 1., 1., 1., 1., 1., 0.], device='cuda:0')\n",
      "tensor([[-1.1079, -2.1684,  0.1284,  ...,  1.2020,  0.9195,  0.1577],\n",
      "        [-2.1634,  0.1058, -0.3701,  ..., -0.3451,  0.7439,  2.6525],\n",
      "        [ 0.4092, -1.4576, -0.0421,  ..., -0.2592,  0.2850, -0.8248],\n",
      "        ...,\n",
      "        [-0.5446,  2.2229,  0.6129,  ..., -1.2818,  1.0554,  0.2625],\n",
      "        [-0.5677, -0.5335,  0.1950,  ...,  0.4837,  0.7290, -0.4799],\n",
      "        [ 0.3482,  1.7408, -1.7177,  ...,  0.1224,  0.2435,  0.5641]],\n",
      "       device='cuda:0')\n",
      "tensor([1., 1., 0., 0., 1., 0., 1., 0., 0., 0., 1., 0., 0., 1., 0., 0., 1., 0.,\n",
      "        1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 0., 0.,\n",
      "        0., 0., 0., 1., 0., 1., 1., 0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1.,\n",
      "        0., 1., 0., 0., 1., 1., 0., 0., 1., 0., 1., 0., 1., 1., 0., 1., 1., 0.,\n",
      "        0., 1., 1., 1., 0., 0., 0., 1., 1., 0., 1., 0., 1., 0., 1., 1., 0., 1.,\n",
      "        1., 1., 1., 1., 0., 0., 1., 1., 0., 0.], device='cuda:0')\n",
      "tensor([[ 6.2565e-03, -2.1573e+00,  1.4286e-01,  ..., -1.4922e-01,\n",
      "         -9.1917e-01,  4.0258e-02],\n",
      "        [ 1.9637e-01,  4.4578e-02, -5.5123e-01,  ..., -6.3190e-01,\n",
      "         -4.0944e-01,  1.2840e+00],\n",
      "        [ 3.5850e-02, -4.7482e-01, -7.1440e-01,  ..., -3.4697e-01,\n",
      "          1.7682e+00,  4.7412e-01],\n",
      "        ...,\n",
      "        [ 7.5148e-01, -6.5764e-01, -7.4810e-02,  ..., -1.7304e-01,\n",
      "          7.1775e-01,  1.0815e-03],\n",
      "        [ 8.8575e-01,  8.0297e-01, -9.6278e-01,  ...,  7.1306e-01,\n",
      "         -1.2839e-01,  9.3193e-01],\n",
      "        [ 1.2176e+00, -4.9100e-01, -3.4851e-01,  ...,  1.2550e+00,\n",
      "          4.8797e-01, -7.2962e-01]], device='cuda:0')\n",
      "tensor([1., 0., 1., 1., 1., 0., 1., 0., 1., 0., 0., 0., 0., 1., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 1., 0., 1., 1., 1., 0., 0., 0., 1., 1., 0., 1., 1., 0., 1.,\n",
      "        0., 1., 1., 0., 0., 1., 0., 1., 1., 1., 1., 0., 0., 1., 1., 0., 1., 0.,\n",
      "        1., 0., 0., 0., 1., 1., 0., 0., 0., 1., 1., 1., 1., 1., 0., 1., 0., 0.,\n",
      "        1., 0., 0., 0., 1., 0., 1., 0., 0., 1., 0., 1., 0., 1., 1., 0., 0., 1.,\n",
      "        0., 0., 0., 0., 1., 0., 0., 0., 1., 0.], device='cuda:0')\n",
      "tensor([[ 0.0412,  0.0602,  0.7809,  ..., -0.9101,  0.1559, -0.9546],\n",
      "        [-1.0019,  0.4895,  0.4622,  ...,  0.1300,  0.0495, -1.4553],\n",
      "        [ 1.5528, -0.0257, -0.2784,  ..., -0.8308,  0.7051,  0.1415],\n",
      "        ...,\n",
      "        [-0.7429,  1.0473,  0.1079,  ..., -0.6130, -2.0793,  0.5188],\n",
      "        [-1.2613,  0.1497,  0.0343,  ..., -0.6076,  0.6049,  1.1671],\n",
      "        [-1.0025,  0.2388,  1.5657,  ...,  1.4118, -0.7076, -0.9744]],\n",
      "       device='cuda:0')\n",
      "tensor([0., 1., 0., 1., 0., 0., 0., 0., 1., 1., 1., 0., 1., 0., 1., 0., 1., 0.,\n",
      "        1., 0., 1., 1., 1., 1., 0., 1., 1., 0., 1., 0., 1., 0., 0., 1., 0., 1.,\n",
      "        0., 1., 0., 1., 0., 0., 1., 1., 0., 1., 1., 0., 1., 1., 0., 1., 0., 0.,\n",
      "        1., 0., 0., 0., 1., 0., 0., 0., 1., 0., 1., 0., 0., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 0., 1., 1., 0., 0., 1., 0., 0., 1., 0., 0., 1., 0., 1., 1., 1.,\n",
      "        0., 0., 1., 1., 0., 0., 0., 0., 0., 1.], device='cuda:0')\n",
      "tensor([[-2.0599, -0.4067,  1.2338,  ..., -1.1041,  0.2344, -2.4703],\n",
      "        [ 0.5997,  0.3083, -1.3947,  ...,  1.0471, -0.7188,  3.1736],\n",
      "        [-0.1941,  0.2365, -0.7309,  ..., -0.6771,  0.4074, -0.1336],\n",
      "        ...,\n",
      "        [-0.9404, -0.5852,  0.3112,  ..., -0.7258,  0.1889,  0.8698],\n",
      "        [ 1.2251, -0.1973, -0.3182,  ...,  0.4044,  0.4470,  0.7498],\n",
      "        [-1.7654, -1.6383, -0.7205,  ...,  0.2514,  1.5116, -0.2299]],\n",
      "       device='cuda:0')\n",
      "tensor([1., 0., 1., 1., 0., 0., 1., 1., 1., 1., 1., 1., 0., 1., 1., 0., 1., 0.,\n",
      "        0., 1., 0., 0., 0., 1., 1., 0., 0., 0., 1., 1., 0., 0., 1., 1., 1., 1.,\n",
      "        0., 1., 0., 1., 0., 0., 1., 0., 1., 0., 1., 0., 1., 0., 1., 0., 1., 0.,\n",
      "        1., 0., 0., 1., 1., 0., 0., 1., 1., 1., 0., 0., 0., 0., 1., 0., 1., 1.,\n",
      "        0., 0., 1., 0., 1., 0., 1., 0., 0., 0., 1., 0., 1., 1., 1., 1., 0., 1.,\n",
      "        1., 0., 1., 1., 1., 0., 1., 1., 0., 0.], device='cuda:0')\n",
      "tensor([[-0.3583,  0.2438,  0.6718,  ...,  0.6634, -1.3307,  1.2061],\n",
      "        [ 1.1804,  0.9511, -0.6300,  ..., -3.2899,  0.4851,  0.7218],\n",
      "        [ 0.6460,  1.7691, -0.7447,  ..., -2.0434, -1.1954,  0.1198],\n",
      "        ...,\n",
      "        [-0.0246, -0.9042,  0.4172,  ...,  0.7429,  1.2945,  0.3753],\n",
      "        [-0.9764,  0.2923, -0.0317,  ..., -0.5621, -0.1500,  0.6885],\n",
      "        [-0.6732,  0.4784,  0.9984,  ..., -0.3463,  1.7642,  0.5104]],\n",
      "       device='cuda:0')\n",
      "tensor([0., 1., 0., 1., 1., 0., 1., 1., 1., 1., 0., 0., 1., 1., 1., 1., 0., 0.,\n",
      "        0., 0., 1., 1., 1., 1., 1., 0., 1., 1., 1., 0., 1., 1., 1., 1., 0., 0.,\n",
      "        0., 1., 1., 1., 0., 1., 1., 1., 0., 1., 1., 1., 0., 0., 1., 1., 1., 1.,\n",
      "        0., 0., 1., 1., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 1., 0., 1., 1.,\n",
      "        0., 0., 0., 1., 0., 0., 1., 1., 1., 1., 0., 0., 1., 0., 1., 0., 0., 1.,\n",
      "        0., 0., 1., 1., 0., 0., 0., 1., 0., 1.], device='cuda:0')\n",
      "tensor([[ 0.7910, -0.7958,  0.8021,  ..., -0.5820,  1.4374,  1.8630],\n",
      "        [-0.1118,  0.9886,  1.1192,  ..., -1.3867,  0.2335,  0.1889],\n",
      "        [-1.6660,  0.4758, -0.3336,  ...,  0.0266,  0.0109, -0.1524],\n",
      "        ...,\n",
      "        [ 0.5283,  0.5472,  0.1812,  ..., -0.1846,  0.3432,  1.2799],\n",
      "        [ 0.4659,  1.3984,  0.5897,  ...,  1.1243,  0.7176,  1.0674],\n",
      "        [-0.7283,  0.6492, -0.7611,  ..., -1.0156, -0.0558,  0.7440]],\n",
      "       device='cuda:0')\n",
      "tensor([1., 0., 1., 0., 0., 0., 0., 0., 1., 1., 0., 0., 0., 0., 1., 0., 0., 0.,\n",
      "        1., 1., 1., 0., 1., 0., 0., 0., 1., 1., 0., 1., 0., 0., 1., 1., 1., 0.,\n",
      "        1., 0., 0., 0., 1., 1., 1., 1., 0., 1., 1., 1., 0., 1., 1., 1., 1., 0.,\n",
      "        0., 0., 0., 0., 1., 0., 0., 1., 0., 0., 1., 1., 0., 0., 0., 0., 0., 1.,\n",
      "        0., 0., 0., 1., 1., 1., 1., 0., 1., 0., 0., 0., 1., 1., 0., 0., 0., 0.,\n",
      "        1., 1., 0., 1., 1., 1., 1., 1., 1., 0.], device='cuda:0')\n",
      "tensor([[ 0.5905, -0.0099, -1.1901,  ...,  0.1210, -0.7786,  2.1563],\n",
      "        [-0.0315,  0.5880,  1.4224,  ...,  0.9863,  0.0585,  0.6601],\n",
      "        [-0.0783,  0.5330,  0.9596,  ...,  1.0907, -0.8258,  1.2554],\n",
      "        ...,\n",
      "        [ 1.6806,  2.3852, -1.3641,  ..., -0.7847, -0.4644,  1.8256],\n",
      "        [-1.0147,  0.8393,  0.3338,  ..., -1.7802, -0.1724, -0.9138],\n",
      "        [-0.7847, -0.1223,  0.5328,  ...,  0.4405, -2.0730,  2.5240]],\n",
      "       device='cuda:0')\n",
      "tensor([1., 0., 0., 1., 0., 0., 0., 1., 1., 0., 1., 1., 0., 0., 1., 1., 1., 1.,\n",
      "        0., 1., 0., 0., 1., 0., 1., 0., 1., 1., 0., 1., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 1., 1., 1., 1., 1., 1., 1., 0., 1., 1., 0., 0., 1., 0., 1., 1., 0.,\n",
      "        1., 0., 0., 1., 0., 0., 1., 0., 1., 1., 1., 1., 0., 1., 1., 1., 0., 0.,\n",
      "        0., 0., 0., 1., 1., 1., 0., 1., 0., 1., 1., 1., 0., 0., 1., 1., 1., 0.,\n",
      "        0., 1., 0., 0., 0., 1., 1., 1., 0., 0.], device='cuda:0')\n",
      "tensor([[ 1.2638,  0.9517, -1.2938,  ...,  0.7955,  0.2252, -0.5268],\n",
      "        [-0.9058,  1.3428,  0.7612,  ..., -0.5752, -0.3743,  1.0527],\n",
      "        [ 0.7172,  1.2645,  1.2004,  ...,  0.2854,  0.2992, -0.1020],\n",
      "        ...,\n",
      "        [ 0.8594, -1.3607,  0.1709,  ...,  0.3690, -1.2631, -1.1470],\n",
      "        [-0.6901, -2.5065,  0.0796,  ...,  0.2182,  0.5400,  0.9424],\n",
      "        [ 0.6785,  0.1950,  0.9858,  ...,  0.6854,  0.6794,  2.5542]],\n",
      "       device='cuda:0')\n",
      "tensor([1., 1., 1., 0., 1., 1., 1., 0., 1., 1., 1., 0., 1., 1., 1., 0., 0., 1.,\n",
      "        1., 1., 0., 1., 0., 0., 1., 1., 1., 0., 0., 0., 1., 0., 0., 1., 1., 1.,\n",
      "        1., 1., 0., 0., 0., 0., 1., 1., 1., 1., 1., 0., 0., 1., 1., 0., 0., 0.,\n",
      "        0., 1., 0., 0., 1., 0., 0., 0., 1., 0., 0., 0., 1., 1., 1., 1., 0., 0.,\n",
      "        0., 0., 0., 1., 0., 0., 0., 0., 0., 1., 1., 0., 1., 0., 1., 1., 0., 1.,\n",
      "        1., 0., 1., 0., 1., 1., 0., 1., 0., 1.], device='cuda:0')\n",
      "tensor([[ 1.5604, -0.4311, -0.7271,  ...,  0.4466, -1.0602,  0.1029],\n",
      "        [-0.2207, -1.7642, -0.7442,  ..., -0.0148, -1.3660,  0.4153],\n",
      "        [-0.2543,  0.7985, -0.1231,  ...,  0.1644, -0.1102, -1.2994],\n",
      "        ...,\n",
      "        [ 0.1794, -0.1436,  0.1144,  ...,  1.5462, -0.1632, -1.1427],\n",
      "        [ 2.6420, -1.1088,  2.3353,  ..., -2.3642,  0.4981, -1.9426],\n",
      "        [-1.8093,  0.1476, -0.7856,  ...,  3.1425,  0.4268,  0.2760]],\n",
      "       device='cuda:0')\n",
      "tensor([1., 1., 0., 1., 0., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0.,\n",
      "        0., 0., 1., 1., 1., 1., 1., 1., 1., 0., 1., 1., 0., 0., 0., 1., 1., 1.,\n",
      "        1., 0., 1., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0.,\n",
      "        1., 1., 0., 0., 1., 1., 1., 0., 1., 1., 1., 1., 1., 0., 1., 1., 0., 1.,\n",
      "        0., 0., 1., 1., 1., 1., 1., 0., 1., 0., 1., 1., 1., 1., 1., 1., 1., 0.,\n",
      "        1., 1., 0., 0., 1., 0., 1., 0., 1., 1.], device='cuda:0')\n"
     ]
    }
   ],
   "source": [
    "for i in dataloader:\n",
    "    print(i[0])\n",
    "    print(i[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set-up learning rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BCEWithLogitsLoss()"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch.optim as optim\n",
    "# optimizer = SGD([{'params': model.classifier[0].parameters(), 'lr': 3e-6, 'momentum': 0.9 }, \n",
    "\n",
    "optimizer = optim.Adam([{'params':mlp.first_layer.parameters(), 'lr':1e-2}, \n",
    "                        {'params':mlp.second_layer.parameters(), 'lr':1e-3}],lr = 2e-2)\n",
    "criterion = nn.BCEWithLogitsLoss()\n",
    "\n",
    "mlp.to(device)\n",
    "criterion.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epoch = 5\n",
    "for epoch in range(n_epoch):\n",
    "    for batch in dataloader:\n",
    "                \n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        x, y = batch\n",
    "        \n",
    "        predictions = mlp(x).squeeze() # Getting rid of batch times 1\n",
    "        \n",
    "        loss = criterion(predictions, y)\n",
    "            \n",
    "        loss.backward()\n",
    "        \n",
    "        optimizer.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.01\n",
      "0.001\n"
     ]
    }
   ],
   "source": [
    "for param_group in optimizer.param_groups:\n",
    "    print(param_group['lr'])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'defaults': {'lr': 0.02,\n",
       "  'betas': (0.9, 0.999),\n",
       "  'eps': 1e-08,\n",
       "  'weight_decay': 0,\n",
       "  'amsgrad': False},\n",
       " 'state': defaultdict(dict, {Parameter containing:\n",
       "              tensor([[-0.0803,  0.0802,  0.0560,  ..., -0.0338, -0.0241,  0.0831],\n",
       "                      [-0.0092, -0.0657,  0.0860,  ...,  0.0911, -0.0204, -0.0413],\n",
       "                      [ 0.2514, -0.1719, -0.0718,  ..., -0.1128, -0.0528,  0.1447],\n",
       "                      ...,\n",
       "                      [ 0.0942,  0.0703,  0.0420,  ...,  0.0204, -0.0685, -0.0527],\n",
       "                      [-0.0645, -0.0242, -0.0934,  ...,  0.0526,  0.0162,  0.1657],\n",
       "                      [-0.0109, -0.1492, -0.1180,  ...,  0.0320, -0.1101,  0.0688]],\n",
       "                     device='cuda:0', requires_grad=True): {'step': 50,\n",
       "               'exp_avg': tensor([[ 1.2066e-04, -3.2025e-04,  2.2104e-04,  ..., -1.4491e-04,\n",
       "                         2.1351e-04,  1.9971e-04],\n",
       "                       [-2.1194e-04, -2.2359e-04,  3.3994e-04,  ..., -5.9924e-04,\n",
       "                         2.8740e-04,  8.0055e-04],\n",
       "                       [-7.5279e-04,  4.9984e-04,  3.6786e-04,  ..., -9.0729e-05,\n",
       "                         8.6042e-04,  1.6873e-05],\n",
       "                       ...,\n",
       "                       [-7.2111e-05, -1.0142e-04,  4.5666e-05,  ...,  2.5774e-05,\n",
       "                         2.4701e-04,  1.9359e-04],\n",
       "                       [-4.4489e-04, -5.8044e-04, -1.6359e-04,  ..., -4.2705e-04,\n",
       "                        -6.6537e-04, -9.6482e-04],\n",
       "                       [-2.1038e-05,  1.0883e-04,  1.8283e-04,  ..., -2.8989e-05,\n",
       "                         1.2208e-04,  6.2375e-05]], device='cuda:0'),\n",
       "               'exp_avg_sq': tensor([[3.0480e-08, 5.2813e-08, 2.9668e-08,  ..., 3.8462e-08, 2.3101e-08,\n",
       "                        3.5849e-08],\n",
       "                       [2.9350e-07, 4.4420e-07, 2.2984e-07,  ..., 2.6970e-07, 1.8241e-07,\n",
       "                        2.8221e-07],\n",
       "                       [2.6205e-07, 2.8131e-07, 2.2891e-07,  ..., 2.1927e-07, 3.4504e-07,\n",
       "                        3.8736e-07],\n",
       "                       ...,\n",
       "                       [2.5197e-08, 3.9512e-08, 1.7667e-08,  ..., 2.8026e-08, 2.2809e-08,\n",
       "                        2.2967e-08],\n",
       "                       [4.2245e-07, 6.9049e-07, 3.9260e-07,  ..., 5.5592e-07, 6.5689e-07,\n",
       "                        4.4783e-07],\n",
       "                       [1.5377e-08, 1.8004e-08, 1.2078e-08,  ..., 1.2854e-08, 9.9680e-09,\n",
       "                        1.3140e-08]], device='cuda:0')},\n",
       "              Parameter containing:\n",
       "              tensor([-0.0599, -0.1094, -0.1572,  0.0283,  0.0078, -0.1558,  0.0347, -0.1306,\n",
       "                      -0.1493, -0.0821,  0.1058, -0.0847, -0.1107, -0.1141, -0.1672,  0.0071,\n",
       "                      -0.1783,  0.0713, -0.0231, -0.0392, -0.0624, -0.0063,  0.1934, -0.0363,\n",
       "                      -0.0763, -0.0425, -0.0572, -0.0577,  0.1216,  0.0068, -0.1518, -0.1302,\n",
       "                      -0.1210, -0.1346,  0.0494, -0.1041,  0.0159, -0.0605, -0.0931, -0.0105,\n",
       "                      -0.0926,  0.1482, -0.1213, -0.1499, -0.0963, -0.0777, -0.0767, -0.2236,\n",
       "                      -0.0513, -0.2051], device='cuda:0', requires_grad=True): {'step': 50,\n",
       "               'exp_avg': tensor([ 2.5866e-05,  8.5838e-04,  5.4896e-04, -1.8420e-05,  3.8743e-04,\n",
       "                        1.3056e-04,  1.8045e-04,  4.1866e-05,  8.5758e-04,  1.1357e-04,\n",
       "                        1.9246e-04,  4.8210e-04,  3.1281e-04, -6.6237e-04,  1.7467e-04,\n",
       "                       -5.9631e-05,  1.0306e-04, -1.1219e-03,  6.0132e-05,  3.2289e-05,\n",
       "                        4.3816e-04,  4.3983e-05,  1.2597e-05,  4.0942e-04,  1.5645e-05,\n",
       "                        9.5249e-04, -1.8600e-05,  1.5569e-04, -1.1345e-05, -6.4573e-05,\n",
       "                        2.0673e-04,  4.2623e-04,  7.1081e-04,  4.8810e-05,  1.8488e-04,\n",
       "                        3.8296e-04, -1.2264e-03,  6.7792e-04,  4.4146e-04, -7.4856e-04,\n",
       "                        5.1375e-05, -5.2114e-04,  2.3830e-04,  1.0731e-03, -2.8998e-04,\n",
       "                        6.9901e-04,  5.8272e-04,  1.9311e-04,  3.0887e-04,  1.9851e-04],\n",
       "                      device='cuda:0'),\n",
       "               'exp_avg_sq': tensor([3.0895e-08, 1.9823e-07, 2.1969e-07, 2.9793e-08, 7.3603e-07, 5.2108e-07,\n",
       "                       2.5638e-08, 1.9607e-07, 2.8746e-07, 9.9061e-08, 7.9484e-07, 2.2295e-07,\n",
       "                       6.2538e-07, 3.9870e-07, 5.3051e-08, 1.3835e-08, 2.2937e-07, 7.6725e-07,\n",
       "                       1.0438e-07, 5.1998e-07, 2.4517e-07, 2.0731e-07, 5.7854e-08, 3.7353e-07,\n",
       "                       1.7820e-07, 3.1723e-07, 2.4595e-07, 6.7383e-08, 2.4834e-07, 1.4119e-07,\n",
       "                       1.3265e-07, 8.1373e-08, 2.8824e-07, 2.9850e-09, 3.0718e-07, 7.3423e-08,\n",
       "                       3.8469e-07, 4.1017e-07, 4.2310e-07, 9.2881e-07, 1.6098e-09, 2.1683e-07,\n",
       "                       1.0134e-07, 6.0761e-07, 2.7521e-07, 2.9957e-07, 3.4426e-07, 2.7071e-08,\n",
       "                       5.1136e-07, 1.7132e-08], device='cuda:0')},\n",
       "              Parameter containing:\n",
       "              tensor([[-0.0409, -0.0956, -0.0950,  0.0497,  0.1339, -0.1305,  0.0479, -0.0847,\n",
       "                       -0.1061, -0.0649,  0.1389, -0.0956, -0.1443,  0.1231, -0.0575,  0.0358,\n",
       "                       -0.1011,  0.1353, -0.0756,  0.1188, -0.0933,  0.0924,  0.0585, -0.1146,\n",
       "                        0.0836, -0.0980, -0.0789, -0.0586,  0.1021,  0.0842, -0.0778, -0.0597,\n",
       "                       -0.0953, -0.0176,  0.1102, -0.0561,  0.1201, -0.1396, -0.1130,  0.1638,\n",
       "                        0.0153,  0.1017, -0.0541, -0.1296,  0.1039, -0.1002, -0.0959, -0.0401,\n",
       "                        0.1377, -0.0280]], device='cuda:0', requires_grad=True): {'step': 50,\n",
       "               'exp_avg': tensor([[ 0.0231,  0.0166,  0.0200, -0.0375, -0.0152,  0.0184, -0.0288,  0.0182,\n",
       "                         0.0165,  0.0227, -0.0209,  0.0191,  0.0229, -0.0274,  0.0188, -0.0306,\n",
       "                         0.0174, -0.0294,  0.0294, -0.0271,  0.0201, -0.0313, -0.0249,  0.0164,\n",
       "                        -0.0236,  0.0154,  0.0278,  0.0224, -0.0325, -0.0274,  0.0264,  0.0189,\n",
       "                         0.0237,  0.0206, -0.0298,  0.0208, -0.0265,  0.0247,  0.0246, -0.0286,\n",
       "                        -0.0187, -0.0352,  0.0152,  0.0174, -0.0296,  0.0201,  0.0156,  0.0245,\n",
       "                        -0.0229,  0.0117]], device='cuda:0'),\n",
       "               'exp_avg_sq': tensor([[4.6765e-05, 2.7195e-05, 3.8364e-05, 6.9813e-05, 4.6690e-05, 2.6747e-05,\n",
       "                        5.0060e-05, 3.8420e-05, 2.2511e-05, 3.5833e-05, 5.1226e-05, 3.4229e-05,\n",
       "                        3.9854e-05, 3.9211e-05, 2.7209e-05, 6.2315e-05, 2.6782e-05, 5.8875e-05,\n",
       "                        6.1848e-05, 4.7414e-05, 4.4337e-05, 6.5193e-05, 5.5384e-05, 2.2476e-05,\n",
       "                        4.2847e-05, 4.1061e-05, 5.4267e-05, 4.5951e-05, 6.2843e-05, 5.2216e-05,\n",
       "                        4.0623e-05, 3.7360e-05, 3.8567e-05, 4.6510e-05, 6.9520e-05, 3.7188e-05,\n",
       "                        3.5694e-05, 4.0761e-05, 4.3268e-05, 4.3430e-05, 3.3870e-05, 6.5355e-05,\n",
       "                        3.0741e-05, 3.0632e-05, 4.8026e-05, 4.8010e-05, 3.8991e-05, 4.4059e-05,\n",
       "                        4.6977e-05, 2.7269e-05]], device='cuda:0')},\n",
       "              Parameter containing:\n",
       "              tensor([0.1381], device='cuda:0', requires_grad=True): {'step': 50,\n",
       "               'exp_avg': tensor([-0.0072], device='cuda:0'),\n",
       "               'exp_avg_sq': tensor([6.3534e-05], device='cuda:0')}}),\n",
       " 'param_groups': [{'params': [Parameter containing:\n",
       "    tensor([[-0.0803,  0.0802,  0.0560,  ..., -0.0338, -0.0241,  0.0831],\n",
       "            [-0.0092, -0.0657,  0.0860,  ...,  0.0911, -0.0204, -0.0413],\n",
       "            [ 0.2514, -0.1719, -0.0718,  ..., -0.1128, -0.0528,  0.1447],\n",
       "            ...,\n",
       "            [ 0.0942,  0.0703,  0.0420,  ...,  0.0204, -0.0685, -0.0527],\n",
       "            [-0.0645, -0.0242, -0.0934,  ...,  0.0526,  0.0162,  0.1657],\n",
       "            [-0.0109, -0.1492, -0.1180,  ...,  0.0320, -0.1101,  0.0688]],\n",
       "           device='cuda:0', requires_grad=True), Parameter containing:\n",
       "    tensor([-0.0599, -0.1094, -0.1572,  0.0283,  0.0078, -0.1558,  0.0347, -0.1306,\n",
       "            -0.1493, -0.0821,  0.1058, -0.0847, -0.1107, -0.1141, -0.1672,  0.0071,\n",
       "            -0.1783,  0.0713, -0.0231, -0.0392, -0.0624, -0.0063,  0.1934, -0.0363,\n",
       "            -0.0763, -0.0425, -0.0572, -0.0577,  0.1216,  0.0068, -0.1518, -0.1302,\n",
       "            -0.1210, -0.1346,  0.0494, -0.1041,  0.0159, -0.0605, -0.0931, -0.0105,\n",
       "            -0.0926,  0.1482, -0.1213, -0.1499, -0.0963, -0.0777, -0.0767, -0.2236,\n",
       "            -0.0513, -0.2051], device='cuda:0', requires_grad=True)],\n",
       "   'lr': 0.01,\n",
       "   'betas': (0.9, 0.999),\n",
       "   'eps': 1e-08,\n",
       "   'weight_decay': 0,\n",
       "   'amsgrad': False},\n",
       "  {'params': [Parameter containing:\n",
       "    tensor([[-0.0409, -0.0956, -0.0950,  0.0497,  0.1339, -0.1305,  0.0479, -0.0847,\n",
       "             -0.1061, -0.0649,  0.1389, -0.0956, -0.1443,  0.1231, -0.0575,  0.0358,\n",
       "             -0.1011,  0.1353, -0.0756,  0.1188, -0.0933,  0.0924,  0.0585, -0.1146,\n",
       "              0.0836, -0.0980, -0.0789, -0.0586,  0.1021,  0.0842, -0.0778, -0.0597,\n",
       "             -0.0953, -0.0176,  0.1102, -0.0561,  0.1201, -0.1396, -0.1130,  0.1638,\n",
       "              0.0153,  0.1017, -0.0541, -0.1296,  0.1039, -0.1002, -0.0959, -0.0401,\n",
       "              0.1377, -0.0280]], device='cuda:0', requires_grad=True),\n",
       "    Parameter containing:\n",
       "    tensor([0.1381], device='cuda:0', requires_grad=True)],\n",
       "   'lr': 0.001,\n",
       "   'betas': (0.9, 0.999),\n",
       "   'eps': 1e-08,\n",
       "   'weight_decay': 0,\n",
       "   'amsgrad': False}]}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "optimizer.__dict__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(optimizer.state_dict(), 'optimizer.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer2 = optim.Adam([{'params':mlp.first_layer.parameters(), 'lr':1e-2}, \n",
    "                        {'params':mlp.second_layer.parameters(), 'lr':1e-3}],lr = 2e-2)\n",
    "optimizer2.load_state_dict(torch.load('optimizer.pt'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'defaults': {'lr': 0.02,\n",
       "  'betas': (0.9, 0.999),\n",
       "  'eps': 1e-08,\n",
       "  'weight_decay': 0,\n",
       "  'amsgrad': False},\n",
       " 'state': defaultdict(dict, {Parameter containing:\n",
       "              tensor([[-0.0803,  0.0802,  0.0560,  ..., -0.0338, -0.0241,  0.0831],\n",
       "                      [-0.0092, -0.0657,  0.0860,  ...,  0.0911, -0.0204, -0.0413],\n",
       "                      [ 0.2514, -0.1719, -0.0718,  ..., -0.1128, -0.0528,  0.1447],\n",
       "                      ...,\n",
       "                      [ 0.0942,  0.0703,  0.0420,  ...,  0.0204, -0.0685, -0.0527],\n",
       "                      [-0.0645, -0.0242, -0.0934,  ...,  0.0526,  0.0162,  0.1657],\n",
       "                      [-0.0109, -0.1492, -0.1180,  ...,  0.0320, -0.1101,  0.0688]],\n",
       "                     device='cuda:0', requires_grad=True): {'step': 50,\n",
       "               'exp_avg': tensor([[ 1.2066e-04, -3.2025e-04,  2.2104e-04,  ..., -1.4491e-04,\n",
       "                         2.1351e-04,  1.9971e-04],\n",
       "                       [-2.1194e-04, -2.2359e-04,  3.3994e-04,  ..., -5.9924e-04,\n",
       "                         2.8740e-04,  8.0055e-04],\n",
       "                       [-7.5279e-04,  4.9984e-04,  3.6786e-04,  ..., -9.0729e-05,\n",
       "                         8.6042e-04,  1.6873e-05],\n",
       "                       ...,\n",
       "                       [-7.2111e-05, -1.0142e-04,  4.5666e-05,  ...,  2.5774e-05,\n",
       "                         2.4701e-04,  1.9359e-04],\n",
       "                       [-4.4489e-04, -5.8044e-04, -1.6359e-04,  ..., -4.2705e-04,\n",
       "                        -6.6537e-04, -9.6482e-04],\n",
       "                       [-2.1038e-05,  1.0883e-04,  1.8283e-04,  ..., -2.8989e-05,\n",
       "                         1.2208e-04,  6.2375e-05]], device='cuda:0'),\n",
       "               'exp_avg_sq': tensor([[3.0480e-08, 5.2813e-08, 2.9668e-08,  ..., 3.8462e-08, 2.3101e-08,\n",
       "                        3.5849e-08],\n",
       "                       [2.9350e-07, 4.4420e-07, 2.2984e-07,  ..., 2.6970e-07, 1.8241e-07,\n",
       "                        2.8221e-07],\n",
       "                       [2.6205e-07, 2.8131e-07, 2.2891e-07,  ..., 2.1927e-07, 3.4504e-07,\n",
       "                        3.8736e-07],\n",
       "                       ...,\n",
       "                       [2.5197e-08, 3.9512e-08, 1.7667e-08,  ..., 2.8026e-08, 2.2809e-08,\n",
       "                        2.2967e-08],\n",
       "                       [4.2245e-07, 6.9049e-07, 3.9260e-07,  ..., 5.5592e-07, 6.5689e-07,\n",
       "                        4.4783e-07],\n",
       "                       [1.5377e-08, 1.8004e-08, 1.2078e-08,  ..., 1.2854e-08, 9.9680e-09,\n",
       "                        1.3140e-08]], device='cuda:0')},\n",
       "              Parameter containing:\n",
       "              tensor([-0.0599, -0.1094, -0.1572,  0.0283,  0.0078, -0.1558,  0.0347, -0.1306,\n",
       "                      -0.1493, -0.0821,  0.1058, -0.0847, -0.1107, -0.1141, -0.1672,  0.0071,\n",
       "                      -0.1783,  0.0713, -0.0231, -0.0392, -0.0624, -0.0063,  0.1934, -0.0363,\n",
       "                      -0.0763, -0.0425, -0.0572, -0.0577,  0.1216,  0.0068, -0.1518, -0.1302,\n",
       "                      -0.1210, -0.1346,  0.0494, -0.1041,  0.0159, -0.0605, -0.0931, -0.0105,\n",
       "                      -0.0926,  0.1482, -0.1213, -0.1499, -0.0963, -0.0777, -0.0767, -0.2236,\n",
       "                      -0.0513, -0.2051], device='cuda:0', requires_grad=True): {'step': 50,\n",
       "               'exp_avg': tensor([ 2.5866e-05,  8.5838e-04,  5.4896e-04, -1.8420e-05,  3.8743e-04,\n",
       "                        1.3056e-04,  1.8045e-04,  4.1866e-05,  8.5758e-04,  1.1357e-04,\n",
       "                        1.9246e-04,  4.8210e-04,  3.1281e-04, -6.6237e-04,  1.7467e-04,\n",
       "                       -5.9631e-05,  1.0306e-04, -1.1219e-03,  6.0132e-05,  3.2289e-05,\n",
       "                        4.3816e-04,  4.3983e-05,  1.2597e-05,  4.0942e-04,  1.5645e-05,\n",
       "                        9.5249e-04, -1.8600e-05,  1.5569e-04, -1.1345e-05, -6.4573e-05,\n",
       "                        2.0673e-04,  4.2623e-04,  7.1081e-04,  4.8810e-05,  1.8488e-04,\n",
       "                        3.8296e-04, -1.2264e-03,  6.7792e-04,  4.4146e-04, -7.4856e-04,\n",
       "                        5.1375e-05, -5.2114e-04,  2.3830e-04,  1.0731e-03, -2.8998e-04,\n",
       "                        6.9901e-04,  5.8272e-04,  1.9311e-04,  3.0887e-04,  1.9851e-04],\n",
       "                      device='cuda:0'),\n",
       "               'exp_avg_sq': tensor([3.0895e-08, 1.9823e-07, 2.1969e-07, 2.9793e-08, 7.3603e-07, 5.2108e-07,\n",
       "                       2.5638e-08, 1.9607e-07, 2.8746e-07, 9.9061e-08, 7.9484e-07, 2.2295e-07,\n",
       "                       6.2538e-07, 3.9870e-07, 5.3051e-08, 1.3835e-08, 2.2937e-07, 7.6725e-07,\n",
       "                       1.0438e-07, 5.1998e-07, 2.4517e-07, 2.0731e-07, 5.7854e-08, 3.7353e-07,\n",
       "                       1.7820e-07, 3.1723e-07, 2.4595e-07, 6.7383e-08, 2.4834e-07, 1.4119e-07,\n",
       "                       1.3265e-07, 8.1373e-08, 2.8824e-07, 2.9850e-09, 3.0718e-07, 7.3423e-08,\n",
       "                       3.8469e-07, 4.1017e-07, 4.2310e-07, 9.2881e-07, 1.6098e-09, 2.1683e-07,\n",
       "                       1.0134e-07, 6.0761e-07, 2.7521e-07, 2.9957e-07, 3.4426e-07, 2.7071e-08,\n",
       "                       5.1136e-07, 1.7132e-08], device='cuda:0')},\n",
       "              Parameter containing:\n",
       "              tensor([[-0.0409, -0.0956, -0.0950,  0.0497,  0.1339, -0.1305,  0.0479, -0.0847,\n",
       "                       -0.1061, -0.0649,  0.1389, -0.0956, -0.1443,  0.1231, -0.0575,  0.0358,\n",
       "                       -0.1011,  0.1353, -0.0756,  0.1188, -0.0933,  0.0924,  0.0585, -0.1146,\n",
       "                        0.0836, -0.0980, -0.0789, -0.0586,  0.1021,  0.0842, -0.0778, -0.0597,\n",
       "                       -0.0953, -0.0176,  0.1102, -0.0561,  0.1201, -0.1396, -0.1130,  0.1638,\n",
       "                        0.0153,  0.1017, -0.0541, -0.1296,  0.1039, -0.1002, -0.0959, -0.0401,\n",
       "                        0.1377, -0.0280]], device='cuda:0', requires_grad=True): {'step': 50,\n",
       "               'exp_avg': tensor([[ 0.0231,  0.0166,  0.0200, -0.0375, -0.0152,  0.0184, -0.0288,  0.0182,\n",
       "                         0.0165,  0.0227, -0.0209,  0.0191,  0.0229, -0.0274,  0.0188, -0.0306,\n",
       "                         0.0174, -0.0294,  0.0294, -0.0271,  0.0201, -0.0313, -0.0249,  0.0164,\n",
       "                        -0.0236,  0.0154,  0.0278,  0.0224, -0.0325, -0.0274,  0.0264,  0.0189,\n",
       "                         0.0237,  0.0206, -0.0298,  0.0208, -0.0265,  0.0247,  0.0246, -0.0286,\n",
       "                        -0.0187, -0.0352,  0.0152,  0.0174, -0.0296,  0.0201,  0.0156,  0.0245,\n",
       "                        -0.0229,  0.0117]], device='cuda:0'),\n",
       "               'exp_avg_sq': tensor([[4.6765e-05, 2.7195e-05, 3.8364e-05, 6.9813e-05, 4.6690e-05, 2.6747e-05,\n",
       "                        5.0060e-05, 3.8420e-05, 2.2511e-05, 3.5833e-05, 5.1226e-05, 3.4229e-05,\n",
       "                        3.9854e-05, 3.9211e-05, 2.7209e-05, 6.2315e-05, 2.6782e-05, 5.8875e-05,\n",
       "                        6.1848e-05, 4.7414e-05, 4.4337e-05, 6.5193e-05, 5.5384e-05, 2.2476e-05,\n",
       "                        4.2847e-05, 4.1061e-05, 5.4267e-05, 4.5951e-05, 6.2843e-05, 5.2216e-05,\n",
       "                        4.0623e-05, 3.7360e-05, 3.8567e-05, 4.6510e-05, 6.9520e-05, 3.7188e-05,\n",
       "                        3.5694e-05, 4.0761e-05, 4.3268e-05, 4.3430e-05, 3.3870e-05, 6.5355e-05,\n",
       "                        3.0741e-05, 3.0632e-05, 4.8026e-05, 4.8010e-05, 3.8991e-05, 4.4059e-05,\n",
       "                        4.6977e-05, 2.7269e-05]], device='cuda:0')},\n",
       "              Parameter containing:\n",
       "              tensor([0.1381], device='cuda:0', requires_grad=True): {'step': 50,\n",
       "               'exp_avg': tensor([-0.0072], device='cuda:0'),\n",
       "               'exp_avg_sq': tensor([6.3534e-05], device='cuda:0')}}),\n",
       " 'param_groups': [{'lr': 0.01,\n",
       "   'betas': (0.9, 0.999),\n",
       "   'eps': 1e-08,\n",
       "   'weight_decay': 0,\n",
       "   'amsgrad': False,\n",
       "   'params': [Parameter containing:\n",
       "    tensor([[-0.0803,  0.0802,  0.0560,  ..., -0.0338, -0.0241,  0.0831],\n",
       "            [-0.0092, -0.0657,  0.0860,  ...,  0.0911, -0.0204, -0.0413],\n",
       "            [ 0.2514, -0.1719, -0.0718,  ..., -0.1128, -0.0528,  0.1447],\n",
       "            ...,\n",
       "            [ 0.0942,  0.0703,  0.0420,  ...,  0.0204, -0.0685, -0.0527],\n",
       "            [-0.0645, -0.0242, -0.0934,  ...,  0.0526,  0.0162,  0.1657],\n",
       "            [-0.0109, -0.1492, -0.1180,  ...,  0.0320, -0.1101,  0.0688]],\n",
       "           device='cuda:0', requires_grad=True), Parameter containing:\n",
       "    tensor([-0.0599, -0.1094, -0.1572,  0.0283,  0.0078, -0.1558,  0.0347, -0.1306,\n",
       "            -0.1493, -0.0821,  0.1058, -0.0847, -0.1107, -0.1141, -0.1672,  0.0071,\n",
       "            -0.1783,  0.0713, -0.0231, -0.0392, -0.0624, -0.0063,  0.1934, -0.0363,\n",
       "            -0.0763, -0.0425, -0.0572, -0.0577,  0.1216,  0.0068, -0.1518, -0.1302,\n",
       "            -0.1210, -0.1346,  0.0494, -0.1041,  0.0159, -0.0605, -0.0931, -0.0105,\n",
       "            -0.0926,  0.1482, -0.1213, -0.1499, -0.0963, -0.0777, -0.0767, -0.2236,\n",
       "            -0.0513, -0.2051], device='cuda:0', requires_grad=True)]},\n",
       "  {'lr': 0.001,\n",
       "   'betas': (0.9, 0.999),\n",
       "   'eps': 1e-08,\n",
       "   'weight_decay': 0,\n",
       "   'amsgrad': False,\n",
       "   'params': [Parameter containing:\n",
       "    tensor([[-0.0409, -0.0956, -0.0950,  0.0497,  0.1339, -0.1305,  0.0479, -0.0847,\n",
       "             -0.1061, -0.0649,  0.1389, -0.0956, -0.1443,  0.1231, -0.0575,  0.0358,\n",
       "             -0.1011,  0.1353, -0.0756,  0.1188, -0.0933,  0.0924,  0.0585, -0.1146,\n",
       "              0.0836, -0.0980, -0.0789, -0.0586,  0.1021,  0.0842, -0.0778, -0.0597,\n",
       "             -0.0953, -0.0176,  0.1102, -0.0561,  0.1201, -0.1396, -0.1130,  0.1638,\n",
       "              0.0153,  0.1017, -0.0541, -0.1296,  0.1039, -0.1002, -0.0959, -0.0401,\n",
       "              0.1377, -0.0280]], device='cuda:0', requires_grad=True),\n",
       "    Parameter containing:\n",
       "    tensor([0.1381], device='cuda:0', requires_grad=True)]}]}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "optimizer2.__dict__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scheduler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BCEWithLogitsLoss()"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch.optim as optim\n",
    "optimizer = optim.Adam(params = mlp.parameters(), lr = 0)\n",
    "criterion = nn.BCEWithLogitsLoss()\n",
    "\n",
    "mlp.to(device)\n",
    "criterion.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "print(optimizer.param_groups[0]['lr'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epoch = 3\n",
    "global_steps = 0\n",
    "warm_up_steps = 1000\n",
    "max_learning_rate = 0.01\n",
    "for epoch in range(n_epoch):\n",
    "    for batch in dataloader:\n",
    "        global_steps += 1  \n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        x, y = batch\n",
    "        \n",
    "        predictions = mlp(x).squeeze() # Getting rid of batch times 1\n",
    "        \n",
    "        loss = criterion(predictions, y)\n",
    "            \n",
    "        loss.backward()\n",
    "        \n",
    "        optimizer.step()\n",
    "        if global_steps < 1000:\n",
    "            optimizer.param_groups[0]['lr'] = global_steps*max_learning_rate/warm_up_steps\n",
    "        else:\n",
    "            optimizer.param_groups[0]['lr'] = max_learning_rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'defaults': {'lr': 0,\n",
       "  'betas': (0.9, 0.999),\n",
       "  'eps': 1e-08,\n",
       "  'weight_decay': 0,\n",
       "  'amsgrad': False},\n",
       " 'state': defaultdict(dict, {Parameter containing:\n",
       "              tensor([[-0.0816,  0.0813,  0.0561,  ..., -0.0335, -0.0243,  0.0828],\n",
       "                      [-0.0087, -0.0660,  0.0853,  ...,  0.0914, -0.0205, -0.0428],\n",
       "                      [ 0.2527, -0.1736, -0.0723,  ..., -0.1129, -0.0533,  0.1445],\n",
       "                      ...,\n",
       "                      [ 0.0938,  0.0712,  0.0417,  ...,  0.0199, -0.0689, -0.0538],\n",
       "                      [-0.0638, -0.0231, -0.0943,  ...,  0.0536,  0.0168,  0.1666],\n",
       "                      [-0.0109, -0.1504, -0.1196,  ...,  0.0311, -0.1104,  0.0687]],\n",
       "                     device='cuda:0', requires_grad=True): {'step': 30,\n",
       "               'exp_avg': tensor([[ 1.0543e-04, -2.4661e-04,  9.2098e-05,  ..., -1.4196e-04,\n",
       "                         1.4496e-05,  1.2534e-04],\n",
       "                       [-5.0293e-04,  1.5700e-04,  5.9730e-04,  ..., -7.2543e-04,\n",
       "                         2.4941e-04,  6.6871e-04],\n",
       "                       [-9.2458e-04,  4.4340e-04,  3.0154e-04,  ..., -4.0061e-04,\n",
       "                         1.8933e-04,  2.0591e-04],\n",
       "                       ...,\n",
       "                       [-3.8033e-05, -2.1730e-04,  9.5305e-05,  ..., -6.4877e-05,\n",
       "                         2.1272e-04,  2.2081e-04],\n",
       "                       [-1.8963e-04, -4.6480e-04,  6.6710e-04,  ..., -6.3033e-04,\n",
       "                        -4.9336e-04, -1.1812e-03],\n",
       "                       [-1.5437e-04,  1.0591e-04,  2.1830e-04,  ...,  2.8997e-05,\n",
       "                         1.5313e-04,  8.9141e-06]], device='cuda:0'),\n",
       "               'exp_avg_sq': tensor([[2.2743e-08, 3.9122e-08, 2.9230e-08,  ..., 2.7561e-08, 2.9238e-08,\n",
       "                        3.5140e-08],\n",
       "                       [1.4946e-07, 1.3581e-07, 1.7173e-07,  ..., 1.6076e-07, 1.5513e-07,\n",
       "                        2.0642e-07],\n",
       "                       [1.6941e-07, 1.7116e-07, 1.6809e-07,  ..., 1.4855e-07, 1.1618e-07,\n",
       "                        2.4219e-07],\n",
       "                       ...,\n",
       "                       [1.5875e-08, 2.5791e-08, 2.7779e-08,  ..., 1.9578e-08, 3.8703e-08,\n",
       "                        2.1875e-08],\n",
       "                       [2.5854e-07, 4.6472e-07, 3.3952e-07,  ..., 5.7655e-07, 3.0390e-07,\n",
       "                        5.5496e-07],\n",
       "                       [1.2444e-08, 1.1906e-08, 1.7598e-08,  ..., 7.8100e-09, 1.7362e-08,\n",
       "                        1.4790e-08]], device='cuda:0')},\n",
       "              Parameter containing:\n",
       "              tensor([-0.0606, -0.1112, -0.1583,  0.0282,  0.0083, -0.1561,  0.0344, -0.1308,\n",
       "                      -0.1504, -0.0832,  0.1064, -0.0853, -0.1118, -0.1131, -0.1669,  0.0076,\n",
       "                      -0.1783,  0.0720, -0.0230, -0.0389, -0.0633, -0.0055,  0.1937, -0.0371,\n",
       "                      -0.0752, -0.0441, -0.0571, -0.0577,  0.1210,  0.0075, -0.1520, -0.1307,\n",
       "                      -0.1222, -0.1355,  0.0486, -0.1051,  0.0174, -0.0612, -0.0937, -0.0089,\n",
       "                      -0.0920,  0.1492, -0.1217, -0.1520, -0.0961, -0.0783, -0.0776, -0.2243,\n",
       "                      -0.0517, -0.2058], device='cuda:0', requires_grad=True): {'step': 30,\n",
       "               'exp_avg': tensor([ 5.9638e-05,  1.1790e-03,  7.8476e-04, -8.3998e-05, -4.7274e-04,\n",
       "                        8.4551e-05, -3.1733e-05,  1.9952e-04,  9.4254e-04,  3.8386e-04,\n",
       "                        2.5166e-05,  4.6204e-04,  6.1540e-04, -7.2826e-04,  1.4687e-04,\n",
       "                       -1.2429e-04,  2.7212e-04, -8.0080e-04,  1.3516e-04, -7.0910e-04,\n",
       "                        5.4246e-04, -6.3740e-04, -2.8982e-05,  7.5450e-04, -6.8961e-04,\n",
       "                        1.1711e-03,  1.6769e-06,  3.2047e-04,  1.9481e-04, -5.1137e-04,\n",
       "                        2.3866e-04,  1.9764e-04,  6.9675e-04,  1.0904e-04,  5.0860e-04,\n",
       "                        4.3274e-04, -1.1905e-03,  8.8479e-04,  4.8234e-04, -1.2788e-03,\n",
       "                       -5.7779e-05, -7.3373e-04,  3.1689e-04,  1.0978e-03, -1.0795e-04,\n",
       "                        1.8125e-04,  7.0605e-04,  2.4874e-04, -1.2899e-04,  2.2870e-04],\n",
       "                      device='cuda:0'),\n",
       "               'exp_avg_sq': tensor([4.1427e-08, 2.0724e-07, 1.8737e-07, 5.4364e-08, 3.7433e-07, 1.9512e-07,\n",
       "                       6.4644e-08, 1.0711e-07, 2.4801e-07, 6.6203e-08, 3.9656e-07, 1.8602e-07,\n",
       "                       3.2422e-07, 3.3022e-07, 5.9340e-08, 2.1784e-08, 1.7393e-07, 5.7115e-07,\n",
       "                       1.0823e-07, 3.0098e-07, 1.2067e-07, 1.3092e-07, 7.8427e-08, 2.4383e-07,\n",
       "                       1.5315e-07, 2.5112e-07, 1.1390e-07, 9.1023e-08, 2.5967e-07, 9.8721e-08,\n",
       "                       9.9492e-08, 6.3109e-08, 1.9078e-07, 5.1435e-09, 2.6644e-07, 7.7989e-08,\n",
       "                       3.1012e-07, 3.5544e-07, 2.5842e-07, 5.1166e-07, 4.2492e-09, 2.2010e-07,\n",
       "                       5.3269e-08, 2.7480e-07, 2.2664e-07, 1.4529e-07, 2.3154e-07, 1.9242e-08,\n",
       "                       3.9140e-07, 1.8031e-08], device='cuda:0')},\n",
       "              Parameter containing:\n",
       "              tensor([[-0.0437, -0.0988, -0.0981,  0.0535,  0.1364, -0.1337,  0.0512, -0.0876,\n",
       "                       -0.1093, -0.0685,  0.1421, -0.0988, -0.1475,  0.1263, -0.0609,  0.0395,\n",
       "                       -0.1041,  0.1384, -0.0788,  0.1224, -0.0962,  0.0956,  0.0616, -0.1177,\n",
       "                        0.0869, -0.1005, -0.0822, -0.0620,  0.1054,  0.0876, -0.0809, -0.0628,\n",
       "                       -0.0982, -0.0210,  0.1136, -0.0591,  0.1239, -0.1432, -0.1162,  0.1676,\n",
       "                        0.0188,  0.1050, -0.0571, -0.1326,  0.1073, -0.1035, -0.0981, -0.0435,\n",
       "                        0.1410, -0.0305]], device='cuda:0', requires_grad=True): {'step': 30,\n",
       "               'exp_avg': tensor([[ 0.0283,  0.0177,  0.0238, -0.0508, -0.0251,  0.0207, -0.0403,  0.0236,\n",
       "                         0.0238,  0.0269, -0.0253,  0.0266,  0.0242, -0.0336,  0.0263, -0.0379,\n",
       "                         0.0237, -0.0300,  0.0294, -0.0360,  0.0192, -0.0457, -0.0271,  0.0163,\n",
       "                        -0.0352,  0.0164,  0.0287,  0.0269, -0.0378, -0.0368,  0.0304,  0.0253,\n",
       "                         0.0260,  0.0268, -0.0366,  0.0257, -0.0371,  0.0265,  0.0257, -0.0325,\n",
       "                        -0.0318, -0.0401,  0.0146,  0.0223, -0.0347,  0.0245,  0.0168,  0.0271,\n",
       "                        -0.0325,  0.0169]], device='cuda:0'),\n",
       "               'exp_avg_sq': tensor([[5.1120e-05, 2.2999e-05, 3.5685e-05, 1.0087e-04, 4.7918e-05, 2.6190e-05,\n",
       "                        7.9420e-05, 3.4149e-05, 3.3430e-05, 3.7540e-05, 5.3457e-05, 3.9737e-05,\n",
       "                        3.9364e-05, 6.0301e-05, 3.9877e-05, 6.1612e-05, 3.4342e-05, 6.2489e-05,\n",
       "                        6.2737e-05, 5.0645e-05, 3.2594e-05, 9.7928e-05, 5.9730e-05, 2.2215e-05,\n",
       "                        6.7034e-05, 2.8383e-05, 4.5356e-05, 6.5253e-05, 8.4332e-05, 6.3553e-05,\n",
       "                        5.5672e-05, 3.8263e-05, 4.2570e-05, 4.1370e-05, 6.2325e-05, 5.0816e-05,\n",
       "                        5.5453e-05, 4.7408e-05, 3.7417e-05, 4.9622e-05, 5.3719e-05, 9.1172e-05,\n",
       "                        2.4992e-05, 2.8597e-05, 6.0164e-05, 3.6051e-05, 4.0408e-05, 4.1863e-05,\n",
       "                        5.1517e-05, 2.9175e-05]], device='cuda:0')},\n",
       "              Parameter containing:\n",
       "              tensor([0.1391], device='cuda:0', requires_grad=True): {'step': 30,\n",
       "               'exp_avg': tensor([-0.0100], device='cuda:0'),\n",
       "               'exp_avg_sq': tensor([4.4102e-05], device='cuda:0')}}),\n",
       " 'param_groups': [{'params': [Parameter containing:\n",
       "    tensor([[-0.0816,  0.0813,  0.0561,  ..., -0.0335, -0.0243,  0.0828],\n",
       "            [-0.0087, -0.0660,  0.0853,  ...,  0.0914, -0.0205, -0.0428],\n",
       "            [ 0.2527, -0.1736, -0.0723,  ..., -0.1129, -0.0533,  0.1445],\n",
       "            ...,\n",
       "            [ 0.0938,  0.0712,  0.0417,  ...,  0.0199, -0.0689, -0.0538],\n",
       "            [-0.0638, -0.0231, -0.0943,  ...,  0.0536,  0.0168,  0.1666],\n",
       "            [-0.0109, -0.1504, -0.1196,  ...,  0.0311, -0.1104,  0.0687]],\n",
       "           device='cuda:0', requires_grad=True), Parameter containing:\n",
       "    tensor([-0.0606, -0.1112, -0.1583,  0.0282,  0.0083, -0.1561,  0.0344, -0.1308,\n",
       "            -0.1504, -0.0832,  0.1064, -0.0853, -0.1118, -0.1131, -0.1669,  0.0076,\n",
       "            -0.1783,  0.0720, -0.0230, -0.0389, -0.0633, -0.0055,  0.1937, -0.0371,\n",
       "            -0.0752, -0.0441, -0.0571, -0.0577,  0.1210,  0.0075, -0.1520, -0.1307,\n",
       "            -0.1222, -0.1355,  0.0486, -0.1051,  0.0174, -0.0612, -0.0937, -0.0089,\n",
       "            -0.0920,  0.1492, -0.1217, -0.1520, -0.0961, -0.0783, -0.0776, -0.2243,\n",
       "            -0.0517, -0.2058], device='cuda:0', requires_grad=True), Parameter containing:\n",
       "    tensor([[-0.0437, -0.0988, -0.0981,  0.0535,  0.1364, -0.1337,  0.0512, -0.0876,\n",
       "             -0.1093, -0.0685,  0.1421, -0.0988, -0.1475,  0.1263, -0.0609,  0.0395,\n",
       "             -0.1041,  0.1384, -0.0788,  0.1224, -0.0962,  0.0956,  0.0616, -0.1177,\n",
       "              0.0869, -0.1005, -0.0822, -0.0620,  0.1054,  0.0876, -0.0809, -0.0628,\n",
       "             -0.0982, -0.0210,  0.1136, -0.0591,  0.1239, -0.1432, -0.1162,  0.1676,\n",
       "              0.0188,  0.1050, -0.0571, -0.1326,  0.1073, -0.1035, -0.0981, -0.0435,\n",
       "              0.1410, -0.0305]], device='cuda:0', requires_grad=True), Parameter containing:\n",
       "    tensor([0.1391], device='cuda:0', requires_grad=True)],\n",
       "   'lr': 0.0003,\n",
       "   'betas': (0.9, 0.999),\n",
       "   'eps': 1e-08,\n",
       "   'weight_decay': 0,\n",
       "   'amsgrad': False}]}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "optimizer.__dict__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
